Import packages and functions¶
In [1]:
# Import packages
%matplotlib inline
from PIL import Image
import numpy as np
import os
from skimage.color import gray2rgb
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
!pip install tensorflow
!pip install keras
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, GaussianNoise, BatchNormalization, GlobalAveragePooling2D
from keras.layers import Conv2D, MaxPooling2D
from keras import Sequential
from keras.optimizers import Adam
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
from keras.preprocessing import image
from keras.models import Model
from keras import backend as K
from sklearn.metrics import confusion_matrix
!pip install git+https://github.com/raghakot/keras-vis.git --upgrade
from vis.visualization import visualize_cam, visualize_saliency, overlay
from keras import activations
from matplotlib import pyplot as plt
import matplotlib.cm as cm
import zipfile
from keras.models import model_from_json
import matplotlib as mpl
FIRST PART: DATA INGESTION¶
Import original images from local computer¶
We used Magnetic Resonance Imaging (MRI) scans from 114 patients with tuberous sclerosis complex (TSC) and from 114 patients with structurally normal MRI (controls).
For each MRI, we manually selected representative axial T2 and T2 FLAIR slices with tubers (in patients with TSC) and with normal findings (in controls). These axial slices were converted to deidentified .jpg images.
We created three folders per TSC and three folders for controls: TSCtrain (566 images), TSCval (130 images), and TSCtest (210 images) and Controltrain (561 images), Controlval (118 images), and Controltest (226 images). Individual patients belonged to only one of the categories (none of the patients had images in different folders).
For the model development part done in a cloud computer environment we only used the train and validation subset. We selected the model with lowest binary cross-entropy error in the validation set as the best model. The best model (InceptionV3) was saved and its performance was evaluated in the local computer on the test set (data not seen previously by the model).
Finally, we visualized the model with class activation maps and saliency maps. To keep a good image visualization size and resolution and with the memory limits of matplotlib.pyplot in jupyter notebooks we divided the images to visualize in three batches in three different jupyter notebooks: I (images from patients 1-8 and controls 1-8), II (images from patients 9-16 and controls 9-16), and III (images from patients 17-25 and controls 17-25).
In [2]:
# Set the figure size
mpl.rcParams['figure.figsize'] = (16,10)
In [3]:
# Unzip files
with zipfile.ZipFile("Controltest1725.zip","r") as zip_ref:
zip_ref.extractall()
with zipfile.ZipFile("TSCtest1725.zip","r") as zip_ref:
zip_ref.extractall()
Path to original images folder¶
In [4]:
# Path to the folder with the original images
pathtoimagesControltest = './Controltest1725/'
pathtoimagesTSCtest = './TSCtest1725/'
SECOND PART: IMPORTATION OF FINAL DATA¶
Import images and labels for the test set¶
In [5]:
## CONTROLS
# Define the image size
image_size = (224, 224)
# Read in the test images for controls
Controltest_images = []
Controltest_dir = pathtoimagesControltest
Controltest_files = os.listdir(Controltest_dir)
# For each image
for f in Controltest_files:
# Open the image
img = Image.open(Controltest_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
Controltest_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
Controltest_X = np.array(Controltest_images)
# Create an array of labels (0 for controls)
Controltest_y = np.array([[0]*Controltest_X.shape[0]]).T
## TSC
# Read in the test images for TSC
TSCtest_images = []
TSCtest_dir = pathtoimagesTSCtest
TSCtest_files = os.listdir(TSCtest_dir)
# For each image
for f in TSCtest_files:
# Open the image
img = Image.open(TSCtest_dir + f)
# Resize the image so that it has a size 224x224
img = img.resize(image_size)
# Transform into a numpy array
img_arr = np.array(img)
# Transform from 224x224 to 224x224x3
if img_arr.shape == image_size:
img_arr = np.expand_dims(img_arr, 3)
img_arr = gray2rgb(img_arr[:, :, 0])
# Add the image to the array of images
TSCtest_images.append(img_arr)
# After having transformed all images, transform the list into a numpy array
TSCtest_X = np.array(TSCtest_images)
# Create an array of labels (1 for TSC)
TSCtest_y = np.array([[1]*TSCtest_X.shape[0]]).T
## MERGE CONTROLS AND TSC
# Train merge files
test_X = np.concatenate([Controltest_X, TSCtest_X])
test_y = np.vstack((Controltest_y, TSCtest_y))
# GPU expects values to be 32-bit floats
test_X = test_X.astype(np.float32)
# Rescale the values to be between 0 and 1
test_X /= 255.
In [6]:
# Shuffle in unison the test_X and the test_y array (123 is just a random number for reproducibility)
shuffled_test_X, shuffled_test_y = shuffle(test_X, test_y, random_state=123)
In [7]:
shuffled_test_X.shape
Out[7]:
In [8]:
# Example of an image to make sure they were converted right
plt.imshow(shuffled_test_X[0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
plt.show()
In [9]:
shuffled_test_y.shape
Out[9]:
In [10]:
shuffled_test_y[0]
Out[10]:
THIRD PART: VISUALIZE CLASS ACTIVATION MAPS AND SALIENCY MAPS¶
Load the model¶
In [11]:
# load model
json_file = open('InceptionV3.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights("InceptionV3.h5")
In [12]:
# Compile model
model.compile(optimizer = Adam(lr = 0.0005), loss = 'binary_crossentropy', metrics = ['accuracy'])
# Generate predictions on test data in the form of probabilities
testInceptionV3 = model.predict(shuffled_test_X, batch_size = 16)
testInceptionV3
Out[12]:
In [13]:
# Relabel actual outcomes and estimated probabilities
y_true = shuffled_test_y
y_predInceptionV3 = testInceptionV3 > 0.5
Visualize the data¶
In [14]:
# Visualize the structure and layers of the model
model.layers
Out[14]:
In [15]:
# Iterate through the MRIs in test set
print('\n \n' + '\033[1m' + 'EACH ORIGINAL IMAGE IS ANALYZED WITH TWO METHODS: CLASS ACTIVATION MAP (UPPER ROW) AND SALIENCY MAP (LOWER ROW)' + '\033[0m' + '\n')
print('\033[1m' + 'FOR EACH METHOD, THE FIRST IMAGE IS THE ORIGINAL IMAGE, THE SECOND IMAGE IS THE MAP, AND THE THIRD IMAGE IS THE MAP SUPERIMPOSED ON THE ORIGINAL IMAGE WITH A TRANSPARENCY THAT IS PROPORTIONAL TO THE ESTIMATED PROBABILITY OF THE IMAGE HAVING TUBER(S) (HIGHER ESTIMATED PROBABILITIES PRODUCE CLEARLY SEEN MAPS OVERLAID ON THE ORIGINAL IMAGE AND LOWER ESTIMATED PROBABILITIES PRODUCE VERY TRANSPARENT MAPS OVERLAYED ON THE ORIGINAL IMAGE)' + '\033[0m'+ '\n \n \n \n')
for i in range(shuffled_test_X.shape[0]):
# Print spaces to separate from the next image
print('\n \n \n \n \n \n \n \n')
# Print real classification of the image
print('\033[1m' + 'REAL CLASSIFICATION OF THE IMAGE: {}'.format('TSC' if y_true[i][0]==1 else 'NO TSC') + '\033[0m')
# Print model classification and model probability of TSC
print('Model classification of this image: {} \nEstimated probability of tuber(s): {} \n'.format('TSC' if testInceptionV3[i][0]>0.5 else 'NO TSC', testInceptionV3[i][0]))
# Print title
print('\033[1m' + 'CLASS ACTIVATION MAP (UPPER ROW) AND SALIENCY MAP (LOWER ROW)' + '\033[0m')
# Original image
plt.subplot(2,3,1)
plt.imshow(shuffled_test_X[i])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map
plt.subplot(2,3,2)
heat_map = visualize_cam(model, layer_idx=300, filter_indices=None, seed_input=shuffled_test_X[i])
plt.imshow(heat_map)
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map superimposed on original image
plt.subplot(2,3,3)
plt.imshow(shuffled_test_X[i])
plt.imshow(heat_map, alpha = 0.8 * testInceptionV3[i][0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Original image
plt.subplot(2,3,4)
plt.imshow(shuffled_test_X[i])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map
heat_map = visualize_saliency(model, layer_idx=300, filter_indices=None, seed_input=shuffled_test_X[i])
plt.subplot(2,3,5)
plt.imshow(heat_map)
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Heat map superimposed on original image
plt.subplot(2,3,6)
plt.imshow(shuffled_test_X[i])
plt.imshow(heat_map, alpha = 0.8 * testInceptionV3[i][0])
plt.grid(b=None)
plt.xticks([])
plt.yticks([])
# Show the image and close it
plt.show()
plt.close()
